@hypothesi/tauri-mcp-server 0.1.3 → 0.2.0

package/README.md

@@ -11,7 +11,7 @@ A **Model Context Protocol (MCP) server** that enables AI assistants like Claude
 
 | Category | Capabilities |
 |----------|-------------|
-| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling—all via WebSocket |
+| 🎯 **UI Automation** | Screenshots, clicks, typing, scrolling, element finding |
 | 🔍 **IPC Monitoring** | Capture and inspect Tauri IPC calls in real-time |
 | 📱 **Mobile Dev** | Manage Android emulators & iOS simulators |
 | 🛠️ **CLI Integration** | Run any Tauri command (`init`, `dev`, `build`, etc.) |

package/dist/driver/plugin-commands.js

@@ -140,3 +140,32 @@ export async function getBackendState() {
         throw new Error(`Failed to get backend state: ${message}`);
     }
 }
+// ============================================================================
+// Window Management
+// ============================================================================
+export const ListWindowsSchema = z.object({});
+/**
+ * Lists all open webview windows in the Tauri application.
+ */
+export async function listWindows() {
+    try {
+        await connectPlugin();
+        const client = getPluginClient();
+        const response = await client.sendCommand({
+            command: 'list_windows',
+        });
+        if (!response.success) {
+            throw new Error(response.error || 'Unknown error');
+        }
+        const windows = response.data;
+        return JSON.stringify({
+            windows,
+            defaultWindow: 'main',
+            totalCount: windows.length,
+        });
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to list windows: ${message}`);
+    }
+}

package/dist/driver/script-manager.js

@@ -0,0 +1,97 @@
+/**
+ * Script Manager - Manages persistent script injection across page navigations.
+ *
+ * This module provides functions to register, remove, and manage scripts that
+ * should be automatically re-injected when pages load or navigate.
+ *
+ * @internal This module is for internal use only and is not exposed as MCP tools.
+ */
+import { getPluginClient, connectPlugin } from './plugin-client.js';
+/**
+ * Registers a script to be injected into the webview.
+ *
+ * The script will be immediately injected if the page is loaded, and will be
+ * automatically re-injected on subsequent page loads/navigations.
+ *
+ * @param id - Unique identifier for the script
+ * @param type - Type of script ('inline' for code, 'url' for external script)
+ * @param content - The script content (JavaScript code) or URL
+ * @param windowLabel - Optional window label to target
+ * @returns Promise resolving to registration result
+ */
+export async function registerScript(id, type, content, windowLabel) {
+    await connectPlugin();
+    const client = getPluginClient();
+    const response = await client.sendCommand({
+        command: 'register_script',
+        args: { id, type, content, windowLabel },
+    });
+    if (!response.success) {
+        throw new Error(response.error || 'Failed to register script');
+    }
+    return response.data;
+}
+/**
+ * Removes a script from the registry and DOM.
+ *
+ * @param id - The script ID to remove
+ * @param windowLabel - Optional window label to target
+ * @returns Promise resolving to removal result
+ */
+export async function removeScript(id, windowLabel) {
+    await connectPlugin();
+    const client = getPluginClient();
+    const response = await client.sendCommand({
+        command: 'remove_script',
+        args: { id, windowLabel },
+    });
+    if (!response.success) {
+        throw new Error(response.error || 'Failed to remove script');
+    }
+    return response.data;
+}
+/**
+ * Clears all registered scripts from the registry and DOM.
+ *
+ * @param windowLabel - Optional window label to target
+ * @returns Promise resolving to the number of scripts cleared
+ */
+export async function clearScripts(windowLabel) {
+    await connectPlugin();
+    const client = getPluginClient();
+    const response = await client.sendCommand({
+        command: 'clear_scripts',
+        args: { windowLabel },
+    });
+    if (!response.success) {
+        throw new Error(response.error || 'Failed to clear scripts');
+    }
+    return response.data;
+}
+/**
+ * Gets all registered scripts.
+ *
+ * @returns Promise resolving to the list of registered scripts
+ */
+export async function getScripts() {
+    await connectPlugin();
+    const client = getPluginClient();
+    const response = await client.sendCommand({
+        command: 'get_scripts',
+        args: {},
+    });
+    if (!response.success) {
+        throw new Error(response.error || 'Failed to get scripts');
+    }
+    return response.data;
+}
+/**
+ * Checks if a script with the given ID is registered.
+ *
+ * @param id - The script ID to check
+ * @returns Promise resolving to true if the script is registered
+ */
+export async function isScriptRegistered(id) {
+    const { scripts } = await getScripts();
+    return scripts.some((s) => { return s.id === id; });
+}

package/dist/driver/scripts/html2canvas-loader.js

@@ -9,6 +9,8 @@ import { createRequire } from 'module';
 // Use createRequire to resolve the path to html2canvas in node_modules
 const require = createRequire(import.meta.url);
 let html2canvasSource = null;
+/** Script ID used for the html2canvas library in the script registry. */
+export const HTML2CANVAS_SCRIPT_ID = '__mcp_html2canvas__';
 /**
  * Get the html2canvas library source code.
  * Loaded lazily and cached.
@@ -21,13 +23,63 @@ export function getHtml2CanvasSource() {
     }
     return html2canvasSource;
 }
+/**
+ * Build a script that captures a screenshot using html2canvas.
+ * Assumes html2canvas is already loaded (either via script manager or inline).
+ */
+export function buildScreenshotCaptureScript(format, quality) {
+    // Note: This script is wrapped by executeAsyncInWebview, so we don't need an IIFE
+    return `
+      // Get the html2canvas function (may be on window, self, or globalThis)
+      const html2canvasFn = typeof html2canvas !== 'undefined' ? html2canvas :
+                           (typeof window !== 'undefined' && window.html2canvas) ? window.html2canvas :
+                           (typeof self !== 'undefined' && self.html2canvas) ? self.html2canvas :
+                           (typeof globalThis !== 'undefined' && globalThis.html2canvas) ? globalThis.html2canvas : null;
+
+      if (!html2canvasFn) {
+         throw new Error('html2canvas not loaded - function not found on any global');
+      }
+
+      // Capture the entire document
+      const element = document.documentElement;
+      if (!element) {
+         throw new Error('document.documentElement is null');
+      }
+
+      // Configure html2canvas options
+      const options = {
+         backgroundColor: null,
+         scale: window.devicePixelRatio || 1,
+         logging: false,
+         useCORS: true,
+         allowTaint: false,
+         imageTimeout: 5000,
+      };
+
+      // Capture the webview
+      const canvas = await html2canvasFn(element, options);
+      if (!canvas) {
+         throw new Error('html2canvas returned null canvas');
+      }
+
+      // Convert to data URL
+      const mimeType = '${format}' === 'jpeg' ? 'image/jpeg' : 'image/png';
+      const dataUrl = canvas.toDataURL(mimeType, ${quality / 100});
+
+      if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+         throw new Error('canvas.toDataURL returned invalid result: ' + (dataUrl ? dataUrl.substring(0, 50) : 'null'));
+      }
+
+      return dataUrl;
+   `;
+}
 /**
  * Build a script that injects html2canvas and captures a screenshot.
+ * This is the legacy function that inlines the library - kept for fallback.
  */
 export function buildScreenshotScript(format, quality) {
     const html2canvas = getHtml2CanvasSource();
     // Note: This script is wrapped by executeAsyncInWebview, so we don't need an IIFE
-    // The wrapper adds: (async () => { const scriptPromise = (async () => { ...script... })(); ... })()
     return `
       try {
          // Inject html2canvas if not already present
@@ -37,47 +89,7 @@ export function buildScreenshotScript(format, quality) {
             // After loading, html2canvas should be on globalThis/self/window
          }
 
-         // Get the html2canvas function (may be on window, self, or globalThis)
-         const html2canvasFn = typeof html2canvas !== 'undefined' ? html2canvas :
-                              (typeof window !== 'undefined' && window.html2canvas) ? window.html2canvas :
-                              (typeof self !== 'undefined' && self.html2canvas) ? self.html2canvas :
-                              (typeof globalThis !== 'undefined' && globalThis.html2canvas) ? globalThis.html2canvas : null;
-
-         if (!html2canvasFn) {
-            throw new Error('html2canvas failed to load - function not found on any global');
-         }
-
-         // Capture the entire document
-         const element = document.documentElement;
-         if (!element) {
-            throw new Error('document.documentElement is null');
-         }
-
-         // Configure html2canvas options
-         const options = {
-            backgroundColor: null,
-            scale: window.devicePixelRatio || 1,
-            logging: false,
-            useCORS: true,
-            allowTaint: false,
-            imageTimeout: 5000,
-         };
-
-         // Capture the webview
-         const canvas = await html2canvasFn(element, options);
-         if (!canvas) {
-            throw new Error('html2canvas returned null canvas');
-         }
-
-         // Convert to data URL
-         const mimeType = '${format}' === 'jpeg' ? 'image/jpeg' : 'image/png';
-         const dataUrl = canvas.toDataURL(mimeType, ${quality / 100});
-
-         if (!dataUrl || !dataUrl.startsWith('data:image/')) {
-            throw new Error('canvas.toDataURL returned invalid result: ' + (dataUrl ? dataUrl.substring(0, 50) : 'null'));
-         }
-
-         return dataUrl;
+         ${buildScreenshotCaptureScript(format, quality)}
       } catch (screenshotError) {
          // Re-throw with more context
          throw new Error('Screenshot capture failed: ' + (screenshotError.message || String(screenshotError)));

package/dist/driver/webview-executor.js

@@ -1,6 +1,7 @@
 import { z } from 'zod';
 import { getPluginClient, connectPlugin } from './plugin-client.js';
-import { buildScreenshotScript } from './scripts/html2canvas-loader.js';
+import { buildScreenshotScript, buildScreenshotCaptureScript, getHtml2CanvasSource, HTML2CANVAS_SCRIPT_ID, } from './scripts/html2canvas-loader.js';
+import { registerScript, isScriptRegistered } from './script-manager.js';
 /**
  * WebView Executor - Native IPC-based JavaScript execution
  *
@@ -38,16 +39,25 @@ export async function ensureReady() {
 export function resetInitialization() {
     isInitialized = false;
 }
-// ============================================================================
-// Core Execution Functions
-// ============================================================================
 /**
  * Execute JavaScript in the Tauri webview using native IPC via WebSocket.
  *
  * @param script - JavaScript code to execute in the webview context
- * @returns Result of the script execution as a string
+ * @param windowId - Optional window label to target (defaults to "main")
+ * @returns Result of the script execution with window context
+ */
+export async function executeInWebview(script, windowId) {
+    const { result } = await executeInWebviewWithContext(script, windowId);
+    return result;
+}
+/**
+ * Execute JavaScript in the Tauri webview and return window context.
+ *
+ * @param script - JavaScript code to execute in the webview context
+ * @param windowId - Optional window label to target (defaults to "main")
+ * @returns Result of the script execution with window context
  */
-export async function executeInWebview(script) {
+export async function executeInWebviewWithContext(script, windowId) {
     try {
         // Ensure we're fully initialized
         await ensureReady();
@@ -56,22 +66,30 @@ export async function executeInWebview(script) {
         // Use 7s timeout (longer than Rust's 5s) so errors return before Node times out.
         const response = await client.sendCommand({
             command: 'execute_js',
-            args: { script },
+            args: { script, windowLabel: windowId },
         }, 7000);
-        // console.log('executeInWebview response:', JSON.stringify(response));
         if (!response.success) {
             throw new Error(response.error || 'Unknown execution error');
         }
+        // Extract window context from response
+        const windowContext = response.windowContext;
         // Parse and return the result
-        const result = response.data;
-        // console.log('executeInWebview result data:', result, 'type:', typeof result);
-        if (result === null || result === undefined) {
-            return 'null';
+        const data = response.data;
+        let result;
+        if (data === null || data === undefined) {
+            result = 'null';
+        }
+        else if (typeof data === 'string') {
+            result = data;
         }
-        if (typeof result === 'string') {
-            return result;
+        else {
+            result = JSON.stringify(data);
         }
-        return JSON.stringify(result);
+        return {
+            result,
+            windowLabel: windowContext?.windowLabel || 'main',
+            warning: windowContext?.warning,
+        };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
@@ -82,10 +100,11 @@ export async function executeInWebview(script) {
  * Execute async JavaScript in the webview with timeout support.
  *
  * @param script - JavaScript code to execute (can use await)
+ * @param windowId - Optional window label to target (defaults to "main")
  * @param timeout - Timeout in milliseconds (default: 5000)
  * @returns Result of the script execution
  */
-export async function executeAsyncInWebview(script, timeout = 5000) {
+export async function executeAsyncInWebview(script, windowId, timeout = 5000) {
     const wrappedScript = `
       return (async () => {
          const timeoutPromise = new Promise((_, reject) => {
@@ -99,7 +118,7 @@ export async function executeAsyncInWebview(script, timeout = 5000) {
          return await Promise.race([scriptPromise, timeoutPromise]);
       })();
    `;
-    return executeInWebview(wrappedScript);
+    return executeInWebview(wrappedScript, windowId);
 }
 // ============================================================================
 // Console Log Capture System
@@ -187,17 +206,46 @@ export async function clearConsoleLogs() {
    `;
     return executeInWebview(script);
 }
-// ============================================================================
-// Screenshot Functionality
-// ============================================================================
+function formatScreenshotResponse(dataUrl, windowContext) {
+    let result = 'Webview screenshot captured (native)';
+    if (windowContext) {
+        result += ` in window "${windowContext.windowLabel}"`;
+        if (windowContext.warning) {
+            result += `\n\n⚠️ ${windowContext.warning}`;
+        }
+    }
+    result += `:\n\n![Screenshot](${dataUrl})`;
+    return result;
+}
+/**
+ * Prepares the html2canvas script for screenshot capture.
+ * Tries to use the script manager for persistence, falls back to inline injection.
+ */
+async function prepareHtml2canvasScript(format, quality) {
+    try {
+        // Check if html2canvas is already registered
+        const isRegistered = await isScriptRegistered(HTML2CANVAS_SCRIPT_ID);
+        if (!isRegistered) {
+            // Register html2canvas via script manager for persistence across navigations
+            const html2canvasSource = getHtml2CanvasSource();
+            await registerScript(HTML2CANVAS_SCRIPT_ID, 'inline', html2canvasSource);
+        }
+        // Use the capture-only script since html2canvas is now registered
+        return buildScreenshotCaptureScript(format, quality);
+    }
+    catch {
+        // Script manager not available, fall back to inline injection
+        return buildScreenshotScript(format, quality);
+    }
+}
 /**
  * Capture a screenshot of the entire webview.
  *
- * @param format - Image format: 'png' or 'jpeg'
- * @param quality - JPEG quality (0-100), only used for jpeg format
+ * @param options - Screenshot options (format, quality, windowId)
  * @returns Base64-encoded image data URL
  */
-export async function captureScreenshot(format = 'png', quality = 90) {
+export async function captureScreenshot(options = {}) {
+    const { format = 'png', quality = 90, windowId } = options;
     // Primary implementation: Use native platform-specific APIs
     // - macOS: WKWebView takeSnapshot
     // - Windows: WebView2 CapturePreview
@@ -212,18 +260,19 @@ export async function captureScreenshot(format = 'png', quality = 90) {
             args: {
                 format,
                 quality,
+                windowLabel: windowId,
             },
         }, 15000);
-        if (response.success && response.data) {
-            // The native command returns a base64 data URL
-            const dataUrl = response.data;
-            // Validate that we got a real data URL
-            if (dataUrl && dataUrl.startsWith('data:image/')) {
-                return `Webview screenshot captured (native):\n\n![Screenshot](${dataUrl})`;
-            }
+        if (!response.success || !response.data) {
+            throw new Error(response.error || 'Native screenshot returned invalid data');
+        }
+        // The native command returns a base64 data URL
+        const dataUrl = response.data;
+        if (!dataUrl || !dataUrl.startsWith('data:image/')) {
+            throw new Error('Native screenshot returned invalid data');
         }
-        // If we get here, native returned but with invalid data - throw to trigger fallback
-        throw new Error(response.error || 'Native screenshot returned invalid data');
+        // Build response with window context
+        return formatScreenshotResponse(dataUrl, response.windowContext);
     }
     catch (nativeError) {
         // Log the native error for debugging, then fall back
@@ -231,8 +280,8 @@ export async function captureScreenshot(format = 'png', quality = 90) {
         console.error(`Native screenshot failed: ${nativeMsg}, falling back to html2canvas`);
     }
     // Fallback 1: Use html2canvas library for high-quality DOM rendering
-    // The library is bundled from node_modules, not loaded from CDN
-    const html2canvasScript = buildScreenshotScript(format, quality);
+    // Try to use the script manager to register html2canvas for persistence
+    const html2canvasScript = await prepareHtml2canvasScript(format, quality);
     // Fallback: Try Screen Capture API if available
     // Note: This script is wrapped by executeAsyncInWebview, so we don't need an IIFE
     const screenCaptureScript = `
@@ -291,7 +340,7 @@ export async function captureScreenshot(format = 'png', quality = 90) {
    `;
     try {
         // Try html2canvas second (after native APIs)
-        const result = await executeAsyncInWebview(html2canvasScript, 10000); // Longer timeout for library loading
+        const result = await executeAsyncInWebview(html2canvasScript, undefined, 10000); // Longer timeout for library loading
         // Validate that we got a real data URL, not 'null' or empty
         if (result && result !== 'null' && result.startsWith('data:image/')) {
             return `Webview screenshot captured:\n\n![Screenshot](${result})`;